#!/bin/bash
#1、将问题分隔成title和content
#2、分别对title和content进行分词
#3、统计词频
if [ $# -lt 1 ] 
then
    echo "Usage: ./splitQuestions srcFile";
    exit 1;
else
    srcFile=$1
fi
if [ $# -eq 2 ];then
    ratio=$2
else
    ratio=3
fi
splitWord_path=~/tools/splitWord
echo "$splitWord_path:$LD_LIBRARY_PATH"
export LD_LIBRARY_PATH="$splitWord_path:$LD_LIBRARY_PATH"
cd $splitWord_path
#对title分词
$splitWord_path/SplitWord  -d "$splitWord_path/all.dic" \
       $srcFile $srcFile".split" 
exit
cd -
awk  -v ratio=$ratio '
BEGIN {
    FS="\t";
    #加载停止词，如果是停止词就不统计

}
{
    split($2, titleArray, " ");
    split($3, contentArray, " ");
    for (id in titleArray) {
        if(cnt[titleArray[id]]) {
            cnt[titleArray[id]] += ratio;
        } else {
            cnt[titleArray[id]] = ratio;
        }
    }
    for (id in contentArray) {
        if(cnt[contentArray[id]]) {
            cnt[contentArray[id]] +=1;
        } else {
            cnt[contentArray[id]] =1;
        }
    }
    printf("%d ", $1);
    for (term in cnt) {
        printf("%s@%d,", term, cnt[term]);
        delete cnt[term];
    }
    printf("\n");
}' $srcFile".split"   
